# -*- coding: UTF-8 -*-
import scrapy
import messAround.util.help as util


# 搜狗热搜
# scrapy crawl sougou_top
class SougouTopSpider(scrapy.Spider):
    name = 'sougou_top'

    allowed_domains = ['sogou.com']

    start_urls = ['https://www.sogou.com']

    headers = {
        'User-Agent': util.agent,
        'Host': 'top.sogou.com',
        'Connection': 'keep-alive',
        'Cache-Control': 'max-age=0',
        'Upgrade-Insecure-Requests': '1',
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
        'Referer': 'http://top.sogou.com/hot/shishi_1.html',
        'Accept-Encoding': 'gzip, deflate',
        'Accept-Language': 'zh-CN,zh;q=0.9',
    }

    meta = {
        'dont_redirect': True,
        'handle_httpstatus_list': [302]
    }

    custom_settings = {
        'ITEM_PIPELINES': {
            'messAround.pipeline.sougou.SougouTopicPipeline': 300
        }
    }

    def start_requests(self):
        urls = [
            'http://top.sogou.com/home.html'
        ]
        for url in urls:
            yield scrapy.Request(url=url, callback=self.parse, headers=self.headers)

    def get_cookie(self, response):
        cookie_url = 'https://www.sogou.com/'
        cookie = response.headers.getlist('Set-Cookie')
        # cookie = response.headers.getlist('Set-Cookie')[0].split(";")[0].split("=")
        # yield scrapy.Request(url=cookie_url, meta=self.meta, callback=self.get_cookie, headers=util.default_headers)

    def parse(self, response):
        for index in range(1, 11):
            item_xpath = f'/html/body/div[4]/div/div[2]/ol/li[{index}]/'
            title = response.xpath(item_xpath + 'a/text()').get()
            link = response.xpath(item_xpath + 'a/@href').get()
            hot_index = response.xpath(item_xpath + 'span[2]/text()').get()
            trend = response.xpath(item_xpath + 'span[2]/i/@class').get()
            yield {
                'source': 1,
                'no': index,
                'title': title,
                'link': link,
                'trend': trend,
                'hot_index': hot_index,
            }
        pass
